/*
 *  crawler.cpp
 *  crawler
 *
 *  Created by Roy Zuo on May 13, 2010.
 *  Copyright 2010 O2 Micro, Inc. All rights reserved.
 *
 */

#include "crawler.hpp"
#include "unique_filename.hpp"

#include <sstream>

#include <boost/algorithm/string.hpp>
#include <boost/random/mersenne_twister.hpp>
#include <boost/random/uniform_int.hpp>
#include <boost/random/variate_generator.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>

Crawler::Crawler ( const string& protocol, const string& host, const vector< string >& paths, const HttpHeaderPairs& headers, const vector< string >& proxies, const vector < string >& expressions )
: _db_connection ( 0 ), _headers ( headers ), _proxies ( proxies )
{
	_seed = new UrlSeed ( "urls.db", protocol, host, paths, expressions );
}

Crawler::~Crawler ()
{
	if ( _db_connection )
	{
		_db_connection->close();
		delete _db_connection;
		_db_connection = 0;
	}

	if ( _seed )
	{
		delete _seed;
		_seed = 0;
	}
}

void Crawler::use_proxy ( string& server, string& port )
{
	unsigned int count = _proxies.size();
	if ( count > 0 )
	{
		boost::mt19937 gen;
		boost::uniform_int<> dist ( 0, count );
		boost::variate_generator<boost::mt19937&, boost::uniform_int<> > die(gen, dist);
		int index = die();

		string proxy = _proxies[ index ];

		boost::regex expr1 ( "^(.*?):(.*?)$" );
		boost::cmatch what;
		if ( boost::regex_match ( proxy.c_str(), what, expr1 ) )
		{
			server = string ( what[1].first, what[1].second );
			port = string ( what[2].first, what[2].second );
		}
	}
}

void Crawler::operator ()()
{
	vector < string > urls;
	_seed->unread_urls ( urls );
	cout << "Found " << urls.size() << " urls ...\n";

	while ( urls.size() > 0 )
	{

		for ( vector< string >::const_iterator iter = urls.begin(); iter != urls.end(); iter ++ )
		{
			boost::posix_time::ptime start_time = boost::posix_time::second_clock::local_time();

			cout << "Reading < " << *iter << " > ...\n";
			string protocol;
			string host;
			string dir;
			string file;

			_seed->split_url ( *iter, protocol, host, dir, file );

			string path = dir + file;

			int retry = 3;

			while ( retry )
			{
				try
				{
					stringstream header_stream;
					stringstream content_stream;
					
					int times = 4 - retry;
					if ( times == 1 )
					{
						cout << "  Trying ... the 1st time ...";
					}
					else if ( times == 2 )
					{
						cout << "  Trying ... the 2nd time ...";
					}
					else if ( times == 3 )
					{
						cout << "  Trying ... the 3rd time ...";
					}
					else
					{
						cout << "  Trying ... the " << times << "st time ...";
					}
					

					if ( retry == 3 )
					{
						read_webpage ( *iter, header_stream, content_stream, host, protocol, host, path, _headers );
					}
					else 
					{
						string server, port;
						use_proxy ( server, port );
						if ( server.length() > 0 && port.length() > 0 )
						{
							read_webpage ( *iter, header_stream, content_stream, host, protocol, host, path, _headers );
						}
						else
						{
							read_webpage ( *iter, header_stream, content_stream, host, protocol, host, path, _headers );
						}
					}

					boost::posix_time::ptime end_time = boost::posix_time::second_clock::local_time();
					boost::posix_time::time_duration td = end_time - start_time;
					cout << " OK (Use Time: " << boost::posix_time::to_simple_string ( td ) << ")\n";
					retry = 0;

				}
				catch ( exception& ex)
				{
					retry -= 1;

					if ( retry == 0 )
					{
						boost::posix_time::ptime end_time = boost::posix_time::second_clock::local_time();
						boost::posix_time::time_duration td = end_time - start_time;
						cout << " Failed ... (Use Time: " << boost::posix_time::to_simple_string ( td ) << ")\n";
						cerr << "\tException Occured: " << ex.what() << " >\n";
						_seed->update( *iter, 99999 );
					}
					else
					{
						cerr << "\tException Occured: " << ex.what() << " >\n";
					}
				}

			}
		}
		urls.clear();
		_seed->unread_urls ( urls );
	}
}

void Crawler::read_webpage ( const string& url, stringstream& header_stream, stringstream& content_stream, const string& server, const string& port, const string& host, const string& path, const HttpHeaderPairs& headers )
{
	boost::asio::io_service io_service;
	WebpageHunter hunter ( io_service, &header_stream, &content_stream, server, port, host, path, _headers );
	io_service.run();

	save_webpage ( url, header_stream.str(), content_stream.str() );
	boost::posix_time::ptime end_time = boost::posix_time::second_clock::local_time();
}

void Crawler::save_webpage ( const string& url, const string& page_header, const string& page_content )
{
	string formatted_page_content = page_content;

	boost::to_lower ( formatted_page_content );
	boost::trim ( formatted_page_content );
	boost::erase_all ( formatted_page_content, "\r" );
	boost::erase_all ( formatted_page_content, "\n" );

	boost::regex ex;
	ex.assign ( "[\\s\\t]+" );

	std::ostringstream t(std::ios::out | std::ios::binary);
	std::ostream_iterator<char> oi(t);
	boost::regex_replace(oi, formatted_page_content.begin(), formatted_page_content.end(), ex, " ", boost::match_default | boost::format_all);
	formatted_page_content = t.str();

	_seed->update ( url, page_header, formatted_page_content );

	create_database();
	update_webpage ( url, page_header, formatted_page_content );
	
}

void Crawler::create_database()
{
	if ( ! _db_connection )
	{
		UniqueFileName uf;
		string filename;

		uf.get_fname ( filename );
		filename += ".db";

		try
		{
			_db_connection = new sqlite3_connection ( filename.c_str() );

			int count = _db_connection->executeint("select count(*) from sqlite_master where name='t_webpages';");
			if(count==0) _db_connection->executenonquery ( SQL_CREATE_WEBPAGES_TABLE );
		}
		catch ( exception &ex )
		{
			cerr << "Exception Occured: " << ex.what() << endl;
		}
	}
}

void Crawler::update_webpage ( const string& url, const string& page_header, const string& page_content )
{
	try
	{
		sqlite3_command st ( *_db_connection, SQL_SELECT_WEBPAGES_COUNT_WITH_URL );
		st.bind ( 1, url.c_str(), url.length() );
		int count = st.executeint();
		if ( count == 0 )
		{
			sqlite3_transaction trans ( *_db_connection );

			{
				sqlite3_command cmd  ( *_db_connection, SQL_INSERT_WEBPAGES );
				cmd.bind ( 1, url.c_str(), url.length() );
				cmd.bind ( 2, page_header.c_str(), page_header.length() );
				cmd.bind ( 3, page_content.c_str(), page_content.length() );

				cmd.executenonquery();
			}

			trans.commit();
		}
		else
		{
			sqlite3_transaction trans ( *_db_connection );

			{
				sqlite3_command cmd  ( *_db_connection, SQL_UPDATE_WEBPAGES );
				cmd.bind ( 1, page_header.c_str(), page_header.length() );
				cmd.bind ( 2, page_content.c_str(), page_content.length() );
				cmd.bind ( 3, url.c_str(), url.length() );

				cmd.executenonquery();
			}

			trans.commit();
		}
	}
	catch ( exception& ex )
	{
		cerr << "Exception Occured: " << ex.what() << endl;
	}
}